import os
import numpy as np
import pandas as pd


train_dir = "/home/datanfs/macong_data/tencent_data/train_preliminary"
test_dir = "/home/datanfs/macong_data/tencent_data/test.zip"

ad_csvf = "ad.csv"
click_csvf = "click_log.csv"
user_csv = "user.csv"
click_uid_csv = "click_udi.csv"
to_stroe_all_csv = "click_uid_ad.csv"

# 对ad.csv进行缺失值处理
def process_ad_csv():
    print("### start process ad_csv ###")
    ad_dir = os.path.join(train_dir, ad_csvf)
    df_ad = pd.read_csv(ad_dir)
    df_ad.loc[df_ad["product_id"] == "\\N"] = 0
    df_ad.loc[df_ad["industry"] == "\\N"] = 0

    print(len(df_ad.loc[df_ad["product_id"] == "\\N"]))
    print(len(df_ad.loc[df_ad["industry"] == "\\N"]))

    df_ad = pd.DataFrame(df_ad, dtype=np.int64)
    print(df_ad.describe())
    return df_ad

# 对click_uid文件进行label处理
def process_click_uid_csv():
    print("### start process click_uid_csv ###")
    df_clickuid = pd.read_csv(os.path.join(train_dir, click_uid_csv))
    df_clickuid["gender"] = df_clickuid["gender"].apply(lambda x: x-1)
    df_clickuid["age"] = df_clickuid["age"].apply(lambda x: x - 1)
    print(df_clickuid.describe())
    return df_clickuid

def merge_ad_clickuid():
    print("### start merge ad click uid ###")
    stroe_dir = os.path.join(train_dir, to_stroe_all_csv)
    print("### 数据存储地址为:", stroe_dir)
    df_ad = process_ad_csv()
    df_clickuid = process_click_uid_csv()
    print("### ad csv columns", df_ad.columns)
    print("### click_uid columns", df_clickuid.columns)

    df_all = pd.merge(df_clickuid, df_ad, on="creative_id", how="left")
    print("------ keys\n", df_all.keys())
    print("------ desc\n", df_all.describe())
    print("------ unique\n", df_all.isnull().any())
    print("------ info\n", df_all.info())
    df_all.to_csv(stroe_dir, index=False)


if __name__ == '__main__':
    # process_ad_csv()
    # process_click_uid_csv()
    merge_ad_clickuid()
